package com.shujia.flink.state;

import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.api.common.serialization.SimpleStringSchema;
import org.apache.flink.api.common.typeinfo.Types;
import org.apache.flink.api.java.tuple.Tuple2;
import org.apache.flink.connector.kafka.source.KafkaSource;
import org.apache.flink.connector.kafka.source.enumerator.initializer.OffsetsInitializer;
import org.apache.flink.runtime.state.hashmap.HashMapStateBackend;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.DataStreamSource;
import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.environment.CheckpointConfig;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;

public class Demo1CheckPoint {
    public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

        //指定状态后端
        //HashMapStateBackend: 将状态保存到TaskManager的内存中
        //EmbeddedRocksDBStateBackend :将状态保存早TaskManager的磁盘中
        env.setStateBackend(new HashMapStateBackend());

        //1、开启checkpoint
        env.enableCheckpointing(20000);

        // 使用 externalized checkpoints，这样 checkpoint 在作业取消后仍就会被保留
        env.getCheckpointConfig().setExternalizedCheckpointCleanup(
                CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION);

        //指定保存快照的位置
        env.getCheckpointConfig().setCheckpointStorage("hdfs://master:9000/flink/checkpoint");


        //创建kafka source
        KafkaSource<String> source = KafkaSource.<String>builder()
                .setBootstrapServers("master:9092,node1:9092,node2:9092")//broker列表
                .setTopics("lines")//指定topic
                .setGroupId("Demo5KafkaSource")//指定消费者组，保证一条数据在一个组内只消费一次
                .setStartingOffsets(OffsetsInitializer.earliest())//指定起始消费的位置
                .setValueOnlyDeserializer(new SimpleStringSchema())
                .build();

        //使用kafka source 创建流（无界流）
        DataStreamSource<String> linesDS = env
                .fromSource(source, WatermarkStrategy.noWatermarks(), "Kafka Source");

        //使用lambda表达式处理数据
        DataStream<String> wordsDS = linesDS
                .flatMap((line, out) -> {
                    for (String word : line.split(",")) {
                        out.collect(word);
                    }
                }, Types.STRING);

        DataStream<Tuple2<String, Integer>> kvDS = wordsDS
                .map(word -> Tuple2.of(word, 1))
                //指定返回类型
                .returns(Types.TUPLE(Types.STRING, Types.INT));

        KeyedStream<Tuple2<String, Integer>, String> keyByDS = kvDS.keyBy(kv -> kv.f0);

        DataStream<Tuple2<String, Integer>> countDS = keyByDS
                .reduce((kv1, kv2) -> Tuple2.of(kv1.f0, kv1.f1 + kv2.f1));

        countDS.print();


        env.execute();
    }
}
